#!/bin/bash

set -o errexit

test_dir=$(realpath $(dirname $0))
. ${test_dir}/../functions

function patch_pvc_request() {
	local cluster=$1
	local size=$2

	echo "Patching PVC request to ${size} in ${cluster}"

	kubectl_bin patch pxc ${cluster} --type=json -p='[{"op": "replace", "path": "/spec/pxc/volumeSpec/persistentVolumeClaim/resources/requests/storage", "value":"'"${size}"'"}]'
}

function get_default_storageclass() {
	kubectl_bin get sc -o jsonpath='{.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
}

function has_error_status() {
	local resize_start=$1
	local resize_finish=$2
	local error_expected=${3:-"false"}

	has_error=$(kubectl_bin get pxc $cluster -o jsonpath='{.status.conditions}' \
		| jq -r --arg start "$resize_start" --arg end "$resize_finish" 'map(select(.type == "error" and (.lastTransitionTime >= $start and .lastTransitionTime <= $end) )) | if length > 0 then "true" else "false" end ')

	if [[ $has_error != $error_expected ]]; then
		echo "The Error status was not $error_expected in resize. Present Statuses:"
		kubectl_bin get pxc $cluster -o jsonpath='{.status}' | jq .
		kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) | tail -n 100
		exit 1
	fi
}

function ensure_default_sc_allows_expansion() {
	local default_sc=$(get_default_storageclass)

	echo "Checking if default storageclass ${default_sc} allows volume expansion"

	local allowVolumeExpansion=$(kubectl_bin get sc -o jsonpath='{.items[?(@.metadata.name=="'"${default_sc}"'")].allowVolumeExpansion}')

	if [[ ${allowVolumeExpansion} != "true" ]]; then
		echo "Default storageclass ${default_sc} does not allow volume expansion"
		exit 0
	fi
}

function apply_resourcequota() {
	local quota=$1
	if [ "$EKS" == 1 -o -n "$OPENSHIFT" ]; then
		local sc='gp2-resizable'
	else
		local sc=$(get_default_storageclass)
	fi

	echo "Applying resourcequota for default storageclass ${sc} with quota ${quota}"

	cat ${test_dir}/conf/resourcequota.yml \
		| sed "s/STORAGECLASS/${sc}/" \
		| sed "s/QUOTA/${quota}/" \
		| kubectl_bin apply -f -
}

function wait_cluster_status() {
	local cluster=$1
	local expected=$2

	echo -n "Waiting for pxc/${cluster} status to be ${expected}"
	until [[ $(kubectl_bin get pxc ${cluster} -o jsonpath='{.status.state}') == ${expected} ]]; do
		if [[ $retry -ge 60 ]]; then
			echo
			echo "pxc/${cluster} did not reach ${expected} status, max retries exceeded"
			exit 1
		fi
		echo -n "."
		sleep 5

		retry=$((retry + 1))
	done

	echo
	echo "pxc/${cluster} status is ${expected}"
}

function wait_all_pvc_resize() {
	local expected_size=$1
	local max_retry=${2:-120}
	local sleep_time=${3:-5}

	for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o name); do
		if ! wait_pvc_resize "$pvc" "$expected_size" "$max_retry" "$sleep_time"; then
			return 1
		fi
	done
	return 0
}

function wait_any_pvc_resize() {
	local expected_size=$1
	local pvc_num=${2:-1}
	local max_retry=${3:-120}
	local sleep_time=${4:-5}

	local retry=0
	echo "Waiting for any of pvces to be resized"
	until [[ $(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o json | jq "[.items[] | select(.status.capacity.storage == \"$expected_size\")] | length") == $pvc_num ]]; do
		if [[ $retry -ge $max_retry ]]; then
			echo
			echo "No pvces were resized, max retries exceeded"
			return 1
		fi
		echo -n "."
		sleep "$sleep_time"

		retry=$((retry + 1))
	done
	echo
	echo "PVC was resized"
	return 0
}

function wait_pvc_resize() {
	local pvc=$1
	local expected_size=$2
	local max_retry=${3:-120}
	local sleep_time=${4:-5}

	local retry=0
	echo "Waiting for $pvc to be resized"
	until [[ $(kubectl_bin get "$pvc" -o jsonpath='{.status.capacity.storage}') == "$expected_size" ]]; do
		if [[ $retry -ge $max_retry ]]; then
			echo
			echo "$pvc was not resized, max retries exceeded"
			return 1
		fi
		echo -n "."
		sleep "$sleep_time"

		retry=$((retry + 1))
	done
	echo
	echo "${pvc} was resized"
	return 0
}

set_debug

if [ "$EKS" == 1 -o -n "$OPENSHIFT" ]; then
	echo "EKS environment detected, creating storageclass for EBS volumes"
	kubectl_bin apply -f ${test_dir}/conf/eks-storageclass.yml
else
	ensure_default_sc_allows_expansion
fi

create_infra ${namespace}

desc 'create first PXC cluster'
cluster="some-name"

if [ "$EKS" == 1 -o -n "$OPENSHIFT" ]; then
	spinup_pxc "${cluster}" "$test_dir/conf/$cluster-eks.yml" "3" "10" "${conf_dir}/secrets.yml"
else
	spinup_pxc "${cluster}" "$test_dir/conf/$cluster.yml" "3" "10" "${conf_dir}/secrets.yml"
fi

desc "test scaling"

patch_pvc_request "${cluster}" "3G"
wait_cluster_consistency "$cluster" 3 2

if wait_all_pvc_resize "3Gi" 120 1; then
	echo "PVC was resized, but resize.expansion is disabled"
	exit 1
fi

echo "Enabling PVC resize"
kubectl_bin patch pxc "${cluster}" --type=json -p='[{"op": "add", "path": "/spec/enableVolumeExpansion", "value":true }]'
sleep 10

resize_start_time=$(TZ=UTC $date +%Y-%m-%dT%H:%M:%SZ)
wait_cluster_consistency "$cluster" 3 2
wait_all_pvc_resize "3Gi"

resize_finish_time=$(TZ=UTC $date +%Y-%m-%dT%H:%M:%SZ)

# There should not be Error status during resize. Check whether there was Error status

has_error_status $resize_start_time $resize_finish_time

if [ "$EKS" == 1 -o -n "$OPENSHIFT" ]; then
	# EKS rate limits PVC expansion for the same EBS volume (1 expand operation in every 6 hours),
	# so we need to delete and recreate the cluster
	echo "Deleting and recreating PXC cluster ${cluster}"
	kubectl_bin delete pxc ${cluster}
	spinup_pxc "${cluster}" "$test_dir/conf/$cluster-eks.yml" "3" "10" "${conf_dir}/secrets.yml"
	echo "Enabling PVC resize for 2nd eks/openshift cluster"
	kubectl_bin patch pxc "${cluster}" --type=json -p='[{"op": "add", "path": "/spec/enableVolumeExpansion", "value":true }]'
	sleep 10
fi

desc 'create 1st insufficient resourcequota'

# We're setting the quota to 10Gi, so we can only resize the first PVC to 4Gi
# the others should fail to resize due to the exceeded quota but operator should
# handle the error and keep the cluster ready. Errored PVCs should keep old size.
apply_resourcequota 10Gi
patch_pvc_request "${cluster}" "4G"
wait_cluster_consistency "$cluster" 3 2
echo

echo "Waiting for any of pvces to be resized"
wait_any_pvc_resize "4Gi"
resized_pvc=$(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o json | jq -r '.items[] | select(.status.capacity.storage == "4Gi") | .metadata.name')
echo "$resized_pvc was resized"
wait_cluster_status ${cluster} "ready"

# Check that 2 PVCs keep old size: 3G
if [[ $(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o json | jq '[.items[] | select(.status.capacity.storage == "3Gi")] | length') != 2 ]]; then
	echo "Number of PVCs with old size is not 2."
	exit 1
fi
for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o json | jq -r '.items[] | select(.status.capacity.storage == "3Gi") | .metadata.name'); do
	if [[ ! $(kubectl_bin describe pvc $pvc | grep "ExceededQuota" | grep "PVC resize failed") ]]; then
		echo "PVCs do not have ExceededQuota error."
		exit 1
	fi
done

desc 'update insufficient resourcequota'
# Check that second resize with reached quota finishes with error, cluster becomes ready
apply_resourcequota 11Gi
patch_pvc_request "${cluster}" "4G"
wait_cluster_consistency "$cluster" 3 2
echo

echo "Waiting for any of pvces to be resized"
wait_any_pvc_resize "4Gi" "2"
resized_pvc=$(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o json | jq -r '.items[] | select(.status.capacity.storage == "4Gi") | .metadata.name')
echo "$resized_pvc was resized"
wait_cluster_status ${cluster} "ready"

# Check that 1 PVC keep old size: 3G
if [[ $(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o json | jq '[.items[] | select(.status.capacity.storage == "3Gi")] | length') != 1 ]]; then
	echo "Number of PVCs with old size is not 1."
	exit 1
fi
for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o json | jq -r '.items[] | select(.status.capacity.storage == "3Gi") | .metadata.name'); do
	if [[ ! $(kubectl_bin describe pvc $pvc | grep "ExceededQuota" | grep "PVC resize failed") ]]; then
		echo "PVCs do not have ExceededQuota error."
		exit 1
	fi
done
desc 'update resourcequota to be sufficient'

# We're setting the quota to 12Gi, so we can resize all PVCs to 4Gi
apply_resourcequota 12Gi
patch_pvc_request "${cluster}" "4G"
wait_cluster_consistency "$cluster" 3 2
echo
wait_all_pvc_resize "4Gi"
wait_cluster_status ${cluster} "ready"

desc "test downscale"

# operator shouldn't try to downscale the PVCs. Error status should be changed to ready.
downscale_start_time=$(TZ=UTC $date +%Y-%m-%dT%H:%M:%SZ)
patch_pvc_request "${cluster}" "1G"
sleep 7
wait_all_pvc_resize "4Gi"
wait_cluster_status ${cluster} "ready"
downscale_finish_time=$(TZ=UTC $date +%Y-%m-%dT%H:%M:%SZ)
has_error_status $downscale_start_time $downscale_finish_time "true"

destroy "${namespace}"
desc "test passed"
